package com.hliushi.crawler;

import org.apache.http.HttpHost;
import org.apache.http.NameValuePair;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;
import org.junit.Test;

import java.util.ArrayList;
import java.util.List;

/**
 * @ClassName HttpClientTest
 * @Description 演示使用HttPClient实现网络爬虫
 * @Author hliushi
 * @Date 2021/6/30 16:39
 * @Version V1.0
 **/
public class HttpClientTest {


    @Test
    public void testGet() throws Exception {
        // 1.创建HttpClient对象
        // new DefaultHttpClient();
        CloseableHttpClient httpClient = HttpClients.createDefault();

        // 2.创建HttpGet请求并进行相关设置
        HttpGet httpGet = new HttpGet("http://www.itcast.cn/?username=java");
        httpGet.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " +
                "(KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36 Edg/91.0.864.59");

        // 3.发起请求
        CloseableHttpResponse response = httpClient.execute(httpGet);

        // 4.判断响应状态码, 并获取响应数据
        if (response.getStatusLine().getStatusCode() == 200) {
            String html = EntityUtils.toString(response.getEntity(), "UTF-8");
            System.out.println("html = " + html);
        }

        // 5.关闭资源
        response.close();
        httpClient.close();
    }

    @Test
    public void testPost() throws Exception {
        // 1.创建HttpClient对象
        // new DefaultHttpClient();
        CloseableHttpClient httpClient = HttpClients.createDefault();

        // 2.创建HttpPost请求并进行相关设置
        HttpPost httpPost = new HttpPost("http://www.itcast.cn/");
        // 准备集合用来存放请求参数
        List<NameValuePair> params = new ArrayList<>();
        params.add(new BasicNameValuePair("username", "hliushi"));
        UrlEncodedFormEntity entity = new UrlEncodedFormEntity(params, "UTF-8");
        httpPost.setEntity(entity);

        httpPost.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " +
                "(KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36 Edg/91.0.864.59");

        // 3.发起请求
        CloseableHttpResponse response = httpClient.execute(httpPost);

        // 4.判断响应状态码, 并获取响应数据
        if (response.getStatusLine().getStatusCode() == 200) {
            String html = EntityUtils.toString(response.getEntity(), "UTF-8");
            System.out.println("html = " + html);
        }

        // 5.关闭资源
        response.close();
        httpClient.close();
    }

    @Test
    public void testPool() throws Exception {
        // 1.创建HttpClient连接管理器
        PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager();

        // 2.设置连接参数
        cm.setMaxTotal(200);            // 设置最大连接数
        cm.setDefaultMaxPerRoute(20);   // 设置每个主机的最大并发

        doGet(cm);
        doGet(cm);
    }


    /**
     * @param
     * @return
     */
    private void doGet(PoolingHttpClientConnectionManager cm) throws Exception {
        // 3.从连接池中获取HttPClient对象
        CloseableHttpClient httpClient = HttpClients.custom().setConnectionManager(cm).build();
        // 在这里加上断点, 观察每次从池中获取一个新的httpClient对象 // 1714  1931

        // 4.创建HttpGet对象
        HttpGet httpGet = new HttpGet("http://www.itcast.cn/");

        // 5.发送请求
        CloseableHttpResponse response = httpClient.execute(httpGet);

        // 6.获取数据
        if (response.getStatusLine().getStatusCode() == 200) {
            String html = EntityUtils.toString(response.getEntity(), "UTF-8");
            System.out.println("html = " + html);
        }

        // 7.关闭资源
        response.close();
        // 注意这里不要关闭HttpClient对象, 因为使用连接池
        // HttpClient对象使用完之后应该会归还连接池中, 而不是关掉
        // httpClient.close();
    }

    @Test
    public void testConfig() throws Exception {
        // 0.创建请求配置对象
        RequestConfig requestConfig = RequestConfig.custom()
                .setSocketTimeout(10000)            // 设置连接超时时间
                .setConnectTimeout(10000)           // 设置创建连接超时时间
                .setConnectionRequestTimeout(10000) // 设置请求超时时间
                .setProxy(new HttpHost("139.224.46.224", 8080))   // 添加代理服务器
                .build();

        // 1.创建HttpClient对象
        // CloseableHttpClient httpClient = HttpClients.createDefault();
        CloseableHttpClient httpClient = HttpClients.custom().setDefaultRequestConfig(requestConfig).build();

        // 2. 创建HttpGet对象
        HttpGet httpGet = new HttpGet("http://www.itcast.cn");
        // 3.发送请求
        CloseableHttpResponse response = httpClient.execute(httpGet);
        // 4.获取响应数据
        if (response.getStatusLine().getStatusCode() == 200) {
            String html = EntityUtils.toString(response.getEntity(), "UTF-8");
            System.out.println("html = " + html);
        }
        // 5.关闭资源
        response.close();
        httpClient.close();
    }
}
